#ifndef COMMON_STREAM_TOKENIZER_H_
#define COMMON_STREAM_TOKENIZER_H_

#include <fermat/common/token.h>
#include <istream>
#include <vector>


namespace fermat {

/*!
* A stream tokenizer splits an input stream
* into a sequence of tokens of different kinds.
* Various token kinds can be registered with
* the tokenizer.
*/
class  StreamTokenizer {
public:

	/*!
	* Creates a StreamTokenizer with no attached stream.
	*/
	StreamTokenizer();
		
	/*!
	* Creates a StreamTokenizer with attached stream.
	*/
	StreamTokenizer(std::istream& istr);
		
	/*!
	* Destroys the StreamTokenizer and deletes all
	* registered tokens.
	*/
	virtual ~StreamTokenizer();

	/*!
	* Attaches the tokenizer to an input stream.
	*/
	void attachToStream(std::istream& istr);

	/*!
	* Adds a token class to the tokenizer. The
	* tokenizer takes ownership of the token and
	* deletes it when no longer needed. Comment
	* and whitespace tokens will be marked as
	* ignorable, which means that next() will not
	* return them.
	*/
	void addToken(Token* pToken);

	/*!
	* Adds a token class to the tokenizer. The
	* tokenizer takes ownership of the token and
	* deletes it when no longer needed.
	* If ignore is true, the token will be marked
	* as ignorable, which means that next() will
	* not return it.
	*/
	void addToken(Token* pToken, bool ignore);

	/*!
	* Extracts the next token from the input stream.
	* Returns a pointer to an EOFToken if there are
	* no more characters to read. 
	* Returns a pointer to an InvalidToken if an
	* invalid character is encountered.
	* If a token is marked as ignorable, it will not
	* be returned, and the next token will be
	* examined.
	* Never returns a NULL pointer.
	* You must not delete the token returned by next().
	*/		
	const Token* next();


private:
	typedef std::vector<Token*> TokenVec;	
	TokenVec       _tokens;
	std::istream  *_pIstr;
	InvalidToken   _invalidToken;
	EOFToken       _eofToken;
};


} 


#endif 